import json
import os
import pandas as pd
import glob
from datetime import datetime

# 读取gold目录下的所有数据文件
def convert_gold_data():
    # 查找所有CSV文件
    gold_files = glob.glob("gold/*.csv")
    
    # 创建知识条目列表
    knowledge_entries = []
    
    # 获取当前最大ID
    try:
        with open("knowledge.json", "r", encoding="utf-8") as f:
            existing_entries = json.load(f)
            if existing_entries:
                max_id = max(entry.get('id', 0) for entry in existing_entries)
                entry_id = max_id + 1
            else:
                entry_id = 1
    except (FileNotFoundError, json.JSONDecodeError):
        entry_id = 1
    
    # 处理不同时间周期的数据文件
    for file_path in gold_files:
        # 从文件名提取时间周期
        file_name = os.path.basename(file_path)
        time_frame = file_name.split('_')[1] if '_' in file_name else "unknown"
        
        print(f"处理{time_frame}黄金价格数据...")
        
        # 读取CSV数据
        try:
            # 尝试使用分号作为分隔符
            df = pd.read_csv(file_path, sep=';')
            
            # 确认数据有效性
            if df.empty or len(df) < 2:
                print(f"警告: {file_path} 数据为空或记录太少，跳过")
                continue
                
            # 提取列名
            columns = df.columns.tolist()
            
            # 确保Date列是日期格式
            if 'Date' in df.columns:
                # 尝试解析日期格式（包含年月日小时）
                df['Date'] = pd.to_datetime(df['Date'], format='%Y.%m.%d %H:%M', errors='coerce')
                
                # 过滤无效日期
                df = df.dropna(subset=['Date'])
                
                # 排序数据
                df = df.sort_values(by='Date')
            
            # 1. 总体摘要
            start_date = df['Date'].min() if 'Date' in df.columns else "未知"
            end_date = df['Date'].max() if 'Date' in df.columns else "未知"
            record_count = len(df)
            
            # 如果能解析为日期，则格式化
            if isinstance(start_date, pd.Timestamp):
                start_date = start_date.strftime('%Y年%m月%d日')
            if isinstance(end_date, pd.Timestamp):
                end_date = end_date.strftime('%Y年%m月%d日')
            
            # 创建整体摘要条目
            summary_text = f"黄金价格数据（XAU/USD）- {time_frame}时间周期。"
            summary_text += f" 数据覆盖从{start_date}到{end_date}的{record_count}条记录。"
            
            # 添加价格范围
            if 'Close' in df.columns:
                min_price = df['Close'].min()
                max_price = df['Close'].max()
                avg_price = df['Close'].mean()
                last_price = df['Close'].iloc[-1]
                
                summary_text += f" 在此期间，黄金价格最低为{min_price:.2f}美元，最高为{max_price:.2f}美元，"
                summary_text += f"平均价格为{avg_price:.2f}美元，最新价格为{last_price:.2f}美元。"
            
            # 添加波动信息
            if 'High' in df.columns and 'Low' in df.columns:
                avg_volatility = (df['High'] - df['Low']).mean()
                summary_text += f" 期间每{time_frame}平均波动幅度为{avg_volatility:.2f}美元。"
            
            # 创建整体摘要的知识条目
            entry = {
                "id": entry_id,
                "text": summary_text
            }
            knowledge_entries.append(entry)
            entry_id += 1
            
            # 2. 按照年份创建摘要
            if 'Date' in df.columns and pd.api.types.is_datetime64_dtype(df['Date']):
                df['Year'] = df['Date'].dt.year
                
                for year, year_data in df.groupby('Year'):
                    # 跳过数据点过少的年份
                    if len(year_data) < 10:
                        continue
                        
                    start_date = year_data['Date'].min().strftime('%Y年%m月%d日')
                    end_date = year_data['Date'].max().strftime('%Y年%m月%d日')
                    
                    yearly_text = f"{year}年黄金价格（XAU/USD）- {time_frame}时间周期。"
                    yearly_text += f" 数据覆盖从{start_date}到{end_date}的{len(year_data)}条记录。"
                    
                    # 添加年度价格信息
                    if 'Close' in year_data.columns:
                        open_price = year_data['Close'].iloc[0]
                        close_price = year_data['Close'].iloc[-1]
                        min_price = year_data['Close'].min()
                        max_price = year_data['Close'].max()
                        
                        yearly_text += f" 年初价格为{open_price:.2f}美元，年末价格为{close_price:.2f}美元，"
                        yearly_text += f"全年涨跌幅为{((close_price-open_price)/open_price*100):.2f}%。"
                        yearly_text += f" 全年最低价格为{min_price:.2f}美元，最高价格为{max_price:.2f}美元。"
                    
                    # 添加关键月份的价格变化
                    if len(year_data) > 30 and 'Date' in year_data.columns and 'Close' in year_data.columns:
                        year_data['Month'] = year_data['Date'].dt.month
                        monthly_data = year_data.groupby('Month')['Close'].agg(['first', 'last'])
                        
                        # 找出变化最大的月份
                        monthly_data['change_pct'] = (monthly_data['last'] - monthly_data['first']) / monthly_data['first'] * 100
                        best_month = monthly_data['change_pct'].idxmax()
                        worst_month = monthly_data['change_pct'].idxmin()
                        
                        # 月份转换为中文
                        month_names = {1: '一月', 2: '二月', 3: '三月', 4: '四月', 5: '五月', 6: '六月', 
                                      7: '七月', 8: '八月', 9: '九月', 10: '十月', 11: '十一月', 12: '十二月'}
                        
                        # 添加月度表现信息
                        if not pd.isna(best_month) and not pd.isna(worst_month):
                            best_change = monthly_data.loc[best_month, 'change_pct']
                            worst_change = monthly_data.loc[worst_month, 'change_pct']
                            
                            yearly_text += f" 表现最好的月份是{month_names.get(best_month, best_month)}，涨幅为{best_change:.2f}%；"
                            yearly_text += f"表现最差的月份是{month_names.get(worst_month, worst_month)}，跌幅为{abs(worst_change):.2f}%。"
                    
                    # 创建年度摘要的知识条目
                    year_entry = {
                        "id": entry_id,
                        "text": yearly_text
                    }
                    knowledge_entries.append(year_entry)
                    entry_id += 1
                
                # 3. 如果是日线或周线数据，为重要时期创建特定摘要
                if time_frame.lower() in ['1d', 'daily', 'day', '1w', 'weekly', 'week']:
                    # 定义重要时期
                    important_periods = [
                        {'name': '2008年金融危机', 'start': '2008-09-01', 'end': '2009-03-31'},
                        {'name': '2011年债务上限危机', 'start': '2011-07-01', 'end': '2011-09-30'},
                        {'name': '2013年黄金暴跌', 'start': '2013-04-01', 'end': '2013-06-30'},
                        {'name': '2016年英国脱欧公投', 'start': '2016-06-01', 'end': '2016-07-31'},
                        {'name': '2020年COVID-19大流行', 'start': '2020-02-01', 'end': '2020-12-31'},
                        {'name': '2022年俄乌冲突', 'start': '2022-02-01', 'end': '2022-12-31'}
                    ]
                    
                    for period in important_periods:
                        try:
                            start_date = pd.to_datetime(period['start'])
                            end_date = pd.to_datetime(period['end'])
                            
                            period_data = df[(df['Date'] >= start_date) & (df['Date'] <= end_date)]
                            
                            # 跳过没有足够数据的时期
                            if len(period_data) < 5:
                                continue
                                
                            period_text = f"{period['name']}期间的黄金价格（XAU/USD）- {time_frame}时间周期。"
                            period_text += f" 数据覆盖从{start_date.strftime('%Y年%m月%d日')}到{end_date.strftime('%Y年%m月%d日')}的{len(period_data)}条记录。"
                            
                            # 添加价格信息
                            if 'Close' in period_data.columns:
                                open_price = period_data['Close'].iloc[0]
                                close_price = period_data['Close'].iloc[-1]
                                min_price = period_data['Close'].min()
                                max_price = period_data['Close'].max()
                                
                                period_text += f" 期初价格为{open_price:.2f}美元，期末价格为{close_price:.2f}美元，"
                                period_text += f"期间涨跌幅为{((close_price-open_price)/open_price*100):.2f}%。"
                                period_text += f" 期间最低价格为{min_price:.2f}美元，最高价格为{max_price:.2f}美元。"
                            
                            # 添加波动信息
                            if 'High' in period_data.columns and 'Low' in period_data.columns:
                                max_daily_change = (period_data['High'] - period_data['Low']).max()
                                period_text += f" 期间单日最大波动为{max_daily_change:.2f}美元。"
                            
                            # 创建特定时期的知识条目
                            period_entry = {
                                "id": entry_id,
                                "text": period_text
                            }
                            knowledge_entries.append(period_entry)
                            entry_id += 1
                        except Exception as e:
                            print(f"警告: 无法处理{period['name']}时期的数据: {str(e)}")
            
            initial_entries = len(knowledge_entries)
            print(f"已处理{time_frame}黄金价格数据，创建了{len(knowledge_entries) - initial_entries}条条目")
            
        except Exception as e:
            print(f"处理{file_path}时出错: {str(e)}")
    
    return knowledge_entries

# 合并与现有知识库
def merge_with_existing_knowledge(new_entries):
    try:
        with open("knowledge.json", "r", encoding="utf-8") as f:
            existing_entries = json.load(f)
    except (FileNotFoundError, json.JSONDecodeError):
        existing_entries = []
    
    # 合并条目
    all_entries = existing_entries + new_entries
    
    # 保存合并后的知识库
    with open("knowledge.json", "w", encoding="utf-8") as f:
        json.dump(all_entries, f, ensure_ascii=False, indent=2)
    
    return len(existing_entries), len(all_entries)

if __name__ == "__main__":
    print("开始转换黄金价格数据...")
    gold_entries = convert_gold_data()
    print(f"共创建{len(gold_entries)}条黄金价格知识条目")
    
    print("开始合并到知识库...")
    existing_count, total_count = merge_with_existing_knowledge(gold_entries)
    print(f"合并完成，知识库从{existing_count}条条目增加到{total_count}条") 